﻿using System;
using System.Collections.Generic;
using System.Text;
using Microsoft.SharePoint;
using Microsoft.SharePoint.Administration;
using System.Net;
using System.Diagnostics;
using Microsoft.SharePoint.Utilities;

namespace Macaw.Wss3.DependencySpiderSolution
{
    public class SPSiteSpider
    {
        #region Private properties
        private DependencyMap dependencymap;
        private AddOrGetResourceDelegate addorgetresource;
        private Queue<string> spiderqueue;
        private List<string> spideredurls;
        private AddDependencyDelegate adddependency;
        private string ExcludeRules;
        private string IncludeRules;
        private bool SpiderEnabled;
        #endregion

        public long SpiderCount;
        public long SpiderData;

        public SPSiteSpider(DependencyMap dependencymap, AddOrGetResourceDelegate addorgetresource, AddDependencyDelegate adddependency)
        {
            this.dependencymap = dependencymap;
            this.addorgetresource = addorgetresource;
            this.adddependency = adddependency;

            // Create a queue for items to be spidered
            spiderqueue = new Queue<string>();

            // Create a list for registering visited urls
            spideredurls = new List<string>();
        }

        private string MakeAbsoluteUrl(string pageurl, string link)
        {
            // Is "Link" already an absolute url?
            if (Uri.IsWellFormedUriString(link, UriKind.Absolute))
            { // Yes, absolute url
                Uri uri = new Uri(link);
                return uri.GetLeftPart(UriPartial.Path);
            }
            else
            { // No, probably relative url
                Uri pageuri = new Uri(pageurl);
                Uri uri = null;
                if (Uri.TryCreate(pageuri, link, out uri))
                {
                    return uri.GetLeftPart(UriPartial.Path);
                }
                else
                {
                    return link;
                }
            }
        }

        #region Spider matching methods

        private class Reference
        {
            public Reference(string Url, string Text)
            {
                this.Url = Url;
                this.Text = Text;
            }

            public string Url;
            public string Text;
        }

        private delegate void MatchDelegate(HtmlAgilityPack.HtmlNode node, List<Reference> matchedlist, string responseurl);

        private void MatchIconReference(HtmlAgilityPack.HtmlNode node, List<Reference> matches, string responseurl)
        {
            HtmlAgilityPack.HtmlAttribute relattr = node.Attributes["rel"];
            if (node.Name.Equals("link", StringComparison.InvariantCultureIgnoreCase) && relattr != null &&
                (relattr.Value.Equals("icon", StringComparison.InvariantCultureIgnoreCase) || relattr.Value.Equals("shortcut icon", StringComparison.InvariantCultureIgnoreCase)))
            {
                HtmlAgilityPack.HtmlAttribute hrefattr = node.Attributes["href"];
                if (hrefattr != null)
                {
                    string hrefvalue = hrefattr.Value;
                    matches.Add(new Reference(MakeAbsoluteUrl(responseurl, hrefvalue), hrefvalue));
                }
            }
        }

        private void MatchCssReference(HtmlAgilityPack.HtmlNode node, List<Reference> matches, string responseurl)
        {
            HtmlAgilityPack.HtmlAttribute relattr = node.Attributes["rel"];
            if (node.Name.Equals("link", StringComparison.InvariantCultureIgnoreCase) && relattr != null && relattr.Value.Equals("stylesheet", StringComparison.InvariantCultureIgnoreCase))
            {
                HtmlAgilityPack.HtmlAttribute hrefattr = node.Attributes["href"];
                if (hrefattr != null)
                {
                    string hrefvalue = hrefattr.Value;
                    matches.Add(new Reference(MakeAbsoluteUrl(responseurl, hrefvalue), hrefvalue));
                }
            }
        }

        private void MatchLinkReference(HtmlAgilityPack.HtmlNode node, List<Reference> matches, string responseurl)
        {
            if (node.Name.Equals("a", StringComparison.InvariantCultureIgnoreCase))
            {
                HtmlAgilityPack.HtmlAttribute hrefattr = node.Attributes["href"];
                if (hrefattr != null)
                {
                    string hrefvalue = hrefattr.Value;
                    if (!hrefvalue.Contains("javascript:"))
                    {
                        matches.Add(new Reference(MakeAbsoluteUrl(responseurl, hrefvalue), node.InnerText) );
                    }
                }
            }
        }

        private void MatchImageReference(HtmlAgilityPack.HtmlNode node, List<Reference> matches, string responseurl)
        {
            if (node.Name.Equals("img", StringComparison.InvariantCultureIgnoreCase))
            {
                HtmlAgilityPack.HtmlAttribute srcattr = node.Attributes["src"];
                if (srcattr != null)
                {
                    string srcvalue = srcattr.Value;
                    string nodetext = node.InnerText;
                    if (string.IsNullOrEmpty(nodetext))
                    {
                        HtmlAgilityPack.HtmlAttribute altattr = node.Attributes["alt"];
                        if (altattr!=null)
                        {
                            nodetext = altattr.Value;
                        }

                        if (string.IsNullOrEmpty(nodetext))
                        {
                            HtmlAgilityPack.HtmlAttribute titleattr = node.Attributes["title"];
                            if (titleattr != null)
                            {
                                nodetext = titleattr.Value;
                            }
                        }
                    }
                    matches.Add(new Reference(MakeAbsoluteUrl(responseurl, srcvalue), nodetext));
                }
            }
        }

        private void MatchScriptReference(HtmlAgilityPack.HtmlNode node, List<Reference> matches, string responseurl)
        {
            if (node.Name.Equals("script", StringComparison.InvariantCultureIgnoreCase))
            {
                HtmlAgilityPack.HtmlAttribute srcattr = node.Attributes["src"];
                if (srcattr != null)
                {
                    string srcvalue = srcattr.Value;
                    matches.Add(new Reference( MakeAbsoluteUrl(responseurl, srcvalue) , srcvalue));
                }
            }
        }

        /// <summary>
        /// This method will search for a specific tagtype and add matches to a list
        /// </summary>
        /// <param name="node">Node</param>
        /// <param name="matches">This list will hold a list of absolute urls</param>
        /// <param name="matchdelegate">The delegate to use to match the nodes</param>
        /// <param name="responseurl">The url of the page/document/node</param>
        private void RecurseNodesAndMatch(HtmlAgilityPack.HtmlNode node, List<Reference> matches, MatchDelegate matchdelegate, string responseurl)
        {
            foreach (HtmlAgilityPack.HtmlNode childnode in node.ChildNodes)
            {
                RecurseNodesAndMatch(childnode, matches, matchdelegate, responseurl);
            }
            if (matchdelegate != null) matchdelegate(node, matches, responseurl);
        }

        private List<Reference> GetIconReferences(string responseurl, HtmlAgilityPack.HtmlDocument document)
        {
            List<Reference> matches = new List<Reference>();
            RecurseNodesAndMatch(document.DocumentNode, matches, new MatchDelegate(MatchIconReference), responseurl);
            return matches;
        }

        private List<Reference> GetCssReferences(string responseurl, HtmlAgilityPack.HtmlDocument document)
        {
            List<Reference> matches = new List<Reference>();
            RecurseNodesAndMatch(document.DocumentNode, matches, new MatchDelegate(MatchCssReference), responseurl);
            return matches;
        }

        private List<Reference> GetImageReferences(string responseurl, HtmlAgilityPack.HtmlDocument document)
        {
            List<Reference> matches = new List<Reference>();
            RecurseNodesAndMatch(document.DocumentNode, matches, new MatchDelegate(MatchImageReference), responseurl);
            return matches;
        }

        private List<Reference> GetScriptReferences(string responseurl, HtmlAgilityPack.HtmlDocument document)
        {
            List<Reference> matches = new List<Reference>();
            RecurseNodesAndMatch(document.DocumentNode, matches, new MatchDelegate(MatchScriptReference), responseurl);
            return matches;
        }

        private List<Reference> GetLinkReferences(string responseurl, HtmlAgilityPack.HtmlDocument document)
        {
            List<Reference> matches = new List<Reference>();
            RecurseNodesAndMatch(document.DocumentNode, matches, new MatchDelegate(MatchLinkReference), responseurl);
            return matches;
        }
        #endregion

        private void IndexSharePointPage(SPFile file, string responseurl, WebResponse response, Queue<string> spiderqueue, List<string> spideredurls, DependencyMap dependencymap)
        {
            Trace.WriteLine(string.Format("{0}: Indexing page: \"{1}\"" , this.GetType().FullName, responseurl));
            Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow resourcerow = addorgetresource(dependencymap, response.ResponseUri.ToString());

            HtmlAgilityPack.HtmlDocument document = new HtmlAgilityPack.HtmlDocument();
            document.Load(response.GetResponseStream());
            List<Reference> iconreferences = GetIconReferences(responseurl, document);
            List<Reference> cssreferences = GetCssReferences(responseurl, document);
            List<Reference> imagereferences = GetImageReferences(responseurl, document);
            List<Reference> scriptreferences = GetScriptReferences(responseurl, document);
            List<Reference> linkreferences = GetLinkReferences(responseurl, document);

            //string responseurl = response.ResponseUri.ToString();

            Trace.WriteLine(string.Format("{0}: Scanning page for links: \"{1}\"", this.GetType().FullName, responseurl));

            #region Add dependencies
            foreach (Reference link in iconreferences)
            {
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow linkresourcerow = addorgetresource(dependencymap, link.Url);
                adddependency(dependencymap, resourcerow, linkresourcerow, LinkType.Icon, link.Text);

                if (!spideredurls.Contains(link.Url) && !spiderqueue.Contains(link.Url) && AllowIndex(link.Url) && SpiderEnabled)
                {
                    spiderqueue.Enqueue(link.Url);
                }
            }

            foreach (Reference link in cssreferences)
            {
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow linkresourcerow = addorgetresource(dependencymap, link.Url);
                adddependency(dependencymap, resourcerow, linkresourcerow, LinkType.Css, link.Text);

                if (!spideredurls.Contains(link.Url) && !spiderqueue.Contains(link.Url) && AllowIndex(link.Url) && SpiderEnabled)
                {
                    spiderqueue.Enqueue(link.Url);
                }
            }

            foreach (Reference link in linkreferences)
            {
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow linkresourcerow = addorgetresource(dependencymap, link.Url);
                adddependency(dependencymap, resourcerow, linkresourcerow, LinkType.Hyperlink, link.Text);

                if (!spideredurls.Contains(link.Url) && !spiderqueue.Contains(link.Url) && AllowIndex(link.Url) && SpiderEnabled)
                {
                    spiderqueue.Enqueue(link.Url);
                }
            }

            foreach (Reference link in scriptreferences)
            {
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow linkresourcerow = addorgetresource(dependencymap, link.Url);
                adddependency(dependencymap, resourcerow, linkresourcerow, LinkType.Script, link.Text);

                if (!spideredurls.Contains(link.Url) && !spiderqueue.Contains(link.Url) && AllowIndex(link.Url) && SpiderEnabled)
                {
                    spiderqueue.Enqueue(link.Url);
                }
            }

            foreach (Reference link in imagereferences)
            {
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow linkresourcerow = addorgetresource(dependencymap, link.Url);
                adddependency(dependencymap, resourcerow, linkresourcerow, LinkType.Image, link.Text);

                if (!spideredurls.Contains(link.Url) && !spiderqueue.Contains(link.Url) && AllowIndex(link.Url) && SpiderEnabled)
                {
                    spiderqueue.Enqueue(link.Url);
                }
            }
            #endregion
        }

        private void IndexSharePointCss(SPFile file, WebResponse response, Queue<string> spiderqueue, List<string> spideredurls, DependencyMap dependencymap)
        {
            // Not yet implemented
        }

        private void IndexSharePointFile(SPFile file, WebResponse response, string responseurl, Queue<string> spiderqueue, List<string> spideredurls, DependencyMap dependencymap)
        {
            try
            {
                string contenttype = response.ContentType.ToLower();
                Trace.WriteLine(string.Format("{0}: SharePoint file: \"{1}\" ({2})", this.GetType().FullName, responseurl, contenttype));
                Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow resourcerow = addorgetresource(dependencymap, responseurl);
                resourcerow.SPID = file.UniqueId;
                resourcerow.Error = false;
                if (contenttype.Contains("text/html"))
                {
                    IndexSharePointPage(file, responseurl, response, spiderqueue, spideredurls, dependencymap);
                }
            }
            catch (Exception ex)
            {
                Trace.WriteLine(string.Format("{0}: Failed to IndexSharePoint File: \"{1}\"", this.GetType().FullName , ex.ToString() ));
                throw;
            }
        }

        private bool AllowIndex(string url)
        {
            bool allow = true;
            if (!string.IsNullOrEmpty(IncludeRules))
            {
                allow = false;
                string[] rules = IncludeRules.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
                foreach (string rule in rules)
                {
                    System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(rule);
                    if (regex.IsMatch(url)) allow = true;
                }
            }

            if (allow && !string.IsNullOrEmpty(ExcludeRules))
            {
                string[] rules = ExcludeRules.Split(new string[] { "\r\n" }, StringSplitOptions.RemoveEmptyEntries);
                foreach (string rule in rules)
                {
                    System.Text.RegularExpressions.Regex regex = new System.Text.RegularExpressions.Regex(rule);
                    if (regex.IsMatch(url)) allow = false;
                }
            }
            if (!allow)
            {
                Trace.Write("Skipped: " + url);
            }
            return allow;
        }

        public void Spider(SPSite sitecollection)
        {
            if (sitecollection.RootWeb.Properties.ContainsKey(DependencySpiderFeatureReceiver.DEPENDENCYSPIDEREXCLUDERULESKEY))
            {
                ExcludeRules = sitecollection.RootWeb.Properties[DependencySpiderFeatureReceiver.DEPENDENCYSPIDEREXCLUDERULESKEY];
            }

            if (sitecollection.RootWeb.Properties.ContainsKey(DependencySpiderFeatureReceiver.DEPENDENCYSPIDERINCLUDERULESKEY))
            {
                IncludeRules = sitecollection.RootWeb.Properties[DependencySpiderFeatureReceiver.DEPENDENCYSPIDERINCLUDERULESKEY];
            }

            if (sitecollection.RootWeb.Properties.ContainsKey(DependencySpiderFeatureReceiver.DEPENDENCYSPIDERSPIDERENABLEDKEY))
            {
                bool.TryParse( sitecollection.RootWeb.Properties[DependencySpiderFeatureReceiver.DEPENDENCYSPIDERSPIDERENABLEDKEY] , out SpiderEnabled );
            }

            foreach (SPWeb web in sitecollection.AllWebs)
            {
                if (web.AllowAnonymousAccess)
                {
                    foreach (SPFolder folder in web.Folders)
                    {
                        foreach (SPFile file in folder.Files)
                        {
                            try
                            {
                                if (file.Item != null && file.Item.ParentList != null && (file.Item.ParentList.AnonymousPermMask64 & SPBasePermissions.ViewListItems) == SPBasePermissions.ViewListItems)
                                {
                                    foreach (SPAlternateUrl alturl in sitecollection.WebApplication.AlternateUrls)
                                    {
                                        string url = SPUrlUtility.CombineUrl(alturl.IncomingUrl, file.ServerRelativeUrl);
                                        if (AllowIndex(url))
                                        {
                                            spiderqueue.Enqueue(url);
                                        }
                                    }
                                }
                            }
                            catch (Exception)
                            {
                                System.Diagnostics.Trace.WriteLine(string.Format("Unable to Spider \"{0}\" in web \"{1}\"", file.Url, web.Url));
                            }
                        }
                    }
                }
            }

            Trace.Write(string.Format("Number or urls fed to the spider: {0}", spiderqueue.Count));

            // Loop while items are queued
            while (spiderqueue.Count > 0)
            {
                // Fetch a url from the spiderqueue
                string spiderurl = spiderqueue.Dequeue();

                // Add the url as spidered. This will prevent it to be respidered.
                if (!spideredurls.Contains(spiderurl))
                {
                    spideredurls.Add(spiderurl);
                }

                try
                {
                    HttpWebRequest request = (HttpWebRequest)HttpWebRequest.Create(spiderurl);
                    request.Proxy = WebRequest.DefaultWebProxy;
                    request.KeepAlive = false;
                    request.Method = "GET";
                    request.ContentType = "application/x-www-form-urlencoded";
                    request.Timeout = 60 * 1000;
                    //request.Credentials = CredentialCache.DefaultCredentials;
                    request.AllowAutoRedirect = true;

                    SpiderCount++;
                    Trace.Write("Spider requesting: " + spiderurl);
                    WebResponse response = request.GetResponse();
                    try
                    {
                        Trace.Write("Spider request result: " + response.ContentType + "  " + response.ContentLength);

                        SpiderData += response.ContentLength;
                        string responseurl = response.ResponseUri.ToString();

                        // Check if the responseurl is different than the original url
                        if (!spiderurl.Equals(responseurl))
                        { // Yes, different
                            if (!spideredurls.Contains(responseurl))
                            {
                                // Add the url to the spidered list
                                spideredurls.Add(responseurl);
                            }
                        }

                        // Chop off the querystring
                        responseurl = response.ResponseUri.GetLeftPart(UriPartial.Path); //Test
                        responseurl = responseurl.Replace("%20", " "); //Test

                        #region Is the reponse a SharePoint object
                        try
                        {
                            using (SPSite responsesite = new SPSite(responseurl))
                            {
                                using (SPWeb responseweb = responsesite.OpenWeb(/*responseurl, false*/))
                                {
                                    SPFile responsefile = null;
                                    if (!responseweb.Exists)
                                    {
                                        responsefile = responsesite.RootWeb.GetFile(responseurl);
                                    }
                                    else
                                    {
                                        responsefile = responseweb.GetFile(responseurl);
                                    }

                                    if (responsefile != null && responsefile.Exists)
                                    {
                                        // Index the SharePoint File
                                        IndexSharePointFile(responsefile, response, responseurl, spiderqueue, spideredurls, dependencymap);

                                        // Also index the SharePoint File which with the original spiderurl
                                        if (!spiderurl.Equals(responseurl))
                                        {
                                            IndexSharePointFile(responsefile, response, spiderurl, spiderqueue, spideredurls, dependencymap);
                                        }
                                    }
                                    else
                                    {
                                        Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow resourcerow = addorgetresource(dependencymap, response.ResponseUri.ToString());
                                        resourcerow.ContentType = response.ContentType;
                                        resourcerow.Error = false;
                                    }

                                }
                            }
                        }
                        catch (System.IO.FileNotFoundException)
                        {
                            //occurs if spsite could not be found. Is valid for external websites
                            Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow resourcerow = addorgetresource(dependencymap, response.ResponseUri.ToString());
                            resourcerow.ContentType = response.ContentType;
                            resourcerow.Error = false;
                        }
                        #endregion
                    }
                    finally
                    {
                        if (response != null) response.Close();
                    }
                }
                catch (Exception ex)
                {
                    Trace.WriteLine(string.Format("{0}: Error \"{1}\" for \"{2}\"", this.GetType().FullName, spiderurl, ex.Message));
                    Macaw.Wss3.DependencySpiderSolution.DependencyMap.ResourceRow resourcerow = addorgetresource(dependencymap, spiderurl);
                    resourcerow.ContentType = ex.Message;
                    resourcerow.Error = true;                    
                }
            }
        }
    }
}
